package com.chaosj.webmagic.process;

import cn.hutool.core.text.StrFormatter;
import com.chaosj.webmagic.dto.BlogHostEnums;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Optional;

/**
 * @className: CsdnScrapeProcess
 * @description:
 * @author: caoyangjie
 * @date: 2022/1/4
 **/
public class CSDNScrapeProcess extends BasePageProcessor{
    // 专栏文章大于 20 篇才 爬取这个 专栏
    public static final int LIMIT_LIST_SIZE = 20;
    // 专栏 每页 40 条
    public static final int PageSize = 40;
    public static final String TOPIC = "topic";

    @Override
    protected String titleXPath() {
        return "//div/h1[@id='articleContentId']/text()";
    }

    @Override
    protected String authorXPath() {
        return "//div//a[@class='follow-nickName vip-name']/text()";
    }

    @Override
    protected String contentXPath() {
        return "//div[@id='article_content']";
    }

    @Override
    protected String errorContentXpath() {
        return "//ul[@class='column_article_list']";
    }

    @Override
    protected void doMoreProcess(Page page, Boolean error) {
        // 抓取 左边专栏列表数据
        if( this.getParams().getBooleanValue("articles") ){
            // 获取 专栏中的分页配置
            page.getHtml().xpath("//div[@class='aside-common-box-content']//li").nodes().stream()
                    .filter(selectable -> doFilterArticle(selectable)).forEach(selectable -> {
                        String articleUri = selectable.xpath("//a/@href").get();
                        page.getTargetRequests().add(new Request(articleUri).setExtras(page.getRequest().getExtras()).putExtra(TOPIC, getTopic(selectable)).putExtra(CUR_DEPTH, this.maxDepth.get()));
                        double pageSize = getPageSize(selectable);
                        for( int i = 2; i < pageSize; i ++ ){
                            Request request = new Request(articleUri.replace(".html", StrFormatter.format("_{}.html",i)));
                            request.setExtras(page.getRequest().getExtras()).putExtra(TOPIC, getTopic(selectable)).putExtra(CUR_DEPTH, this.maxDepth.get());
                            page.getTargetRequests().add(request);
                        }
            });
        }
    }

    private double getPageSize(Selectable selectable) {
        String articleSizeString = Optional.ofNullable(selectable.xpath("//div[@class='special-column-num']/text()").get()).orElse("0").replace("篇","");
        Integer articleSize = Integer.valueOf(articleSizeString);
        return Math.ceil(articleSize/40);
    }

    private String getTopic(Selectable selectable){
        return Optional.ofNullable(selectable.xpath("//span/text()").get()).orElse("");
    }

    private boolean doFilterArticle(Selectable selectable){
        return selectable.xpath("//div[@class='pay-tag']").nodes().size()==0 &&
        Integer.valueOf(Optional.ofNullable(selectable.xpath("//div[@class='special-column-num']/text()").get()).orElse("0").replace("篇","")) > LIMIT_LIST_SIZE;
    }

    @Override
    public Site getSite() {
        return Site.me().setDomain(BlogHostEnums.CSDN.getUrl());
    }
}
